---
title: "IV with Endogenous Markets"
author: "Andrei Munteanu"
date: "27/09/2021"
output:
  html_document:
    fig_width: 8 
    fig_height: 8 
---
<style type="text/css">
.main-container {
  max-width: 1800px;
  margin-left: auto;
  margin-right: auto;
}
</style>

```{r options, include=FALSE, echo=FALSE, cache=FALSE}
knitr::opts_knit$set(root.dir=getwd())
knitr::opts_chunk$set(cache.lazy=FALSE,cache=FALSE,warning=FALSE,echo=TRUE,comment='',prompt=T)
options(modelsummary_format_numeric_latex='plain')
knitr::opts_template$set(
  regular=list(include=TRUE,echo=TRUE,cache=FALSE),
  regular_cache=list(include=TRUE,echo=TRUE,cache=FALSE),
  invisible=list(include=FALSE,echo=FALSE,cache=FALSE,warning=FALSE,results=FALSE),
  muted=list(include=TRUE,echo=TRUE,cache=FALSE,warning=FALSE,results=FALSE),
  latex=list(include=TRUE,echo=TRUE,cache=FALSE))

```

## {.tabset}

### Create Transitions Between Towns

Find transitions between schools

```{r, opts.label='muted'}
data_regression<-data_regression_orig
data_graph<-data_regression %>% 
  filter(!is.na(id_bac) & !is.na(id_adm) & unitate_de_invatamant!='') %>%
  group_by(scoala_de_provenienta,town) %>%
  summarise(n=n(),yrs_ms=length(unique(an)))
data_graph2<-data_graph%>%
  group_by(scoala_de_provenienta) %>%
  mutate(p_ms=n/sum(n),n_ms=sum(n)) %>%
  arrange(scoala_de_provenienta,-p_ms) %>%
  mutate(r_ms=row_number()) %>%
  group_by(town) %>%
  mutate(p_market=n/sum(n),n_market=sum(n)) %>%
  arrange(town,-p_market) %>%
  mutate(r_market=row_number(),n_yr=n/yrs_ms) %>%
  #filter(((p_ms>0.3 | p_market>0.05) & (n_yr>7|n>30)) |(r_market==1 & p_market>0.1 & (n_yr>10 | n>30))|(r_ms==1 & p_ms>0.5 & (n>30 | n_yr>10))) %>% 
  # filter((n_ms>50 & p_ms>0.5) | (n_ms>40 & p_ms>0.6) | (n_ms>30 & p_ms>0.7) | (n_ms>20 & p_ms>0.8) | (p_ms>0.9) |
  #          r_ms==1 | r_market==1 | 
  #          (n_market>50 & p_market>0.1) | p_market>0.2 |(n_market>100 & p_market>0.05) ) %>% 
  # ungroup %>%
  #   filter((n_ms>50 & p_ms>0.4) | (n_ms>40 & p_ms>0.5) | (n_ms>30 & p_ms>0.6) | (n_ms>20 & p_ms>0.7) | (p_ms>0.8) |
  #          r_ms==1 | r_market==1 | 
  #          (n_market>50 & p_market>0.1) | p_market>0.2 |(n_market>100 & p_market>0.05) ) %>% 
  # ungroup %>%
   filter((n_ms>50 & p_ms>0.8) | (n_ms>40 & p_ms>0.85) | (n_ms>30 & p_ms>0.9) | (n_ms>20 & p_ms>0.95) | (p_ms>0.95) |
           r_ms==1 | r_market==1 | 
           (n_market>50 & p_market>0.2) | p_market>0.2 |(n_market>100 & p_market>0.1) ) %>% 
  ungroup %>%
  select(scoala_de_provenienta,town)

graph<-graph_from_data_frame(data_graph2)                            
market<-unname(igraph::components(graph)$membership)
market_name<-names(igraph::components(graph)$membership)
membership<-data.frame("market"=market,"market_name"=market_name) %>% arrange(market)
#unique(igraph::membership$component)
#igraph::components(graph)
```

Merge markets into data

```{r, opts.label='muted'}
data_regression<-base::merge(data_regression,membership,by.x="town",by.y="market_name",all.x=T)
data_regression<-base::merge(data_regression,membership,by.x="scoala_de_provenienta",by.y="market_name",all.x=T,suffixes=c("",".ms"))

#replace places without market with town name
data_regression <- data_regression %>% mutate(market_name=ifelse(is.na(market),town,market))

#assign middle schools with missing markets to the most likely market
missing_schools <-data_regression %>% 
  filter(is.na(market.ms)) %>% 
  group_by(scoala_de_provenienta,market) %>% 
  summarise(n=n()) %>% 
  group_by(scoala_de_provenienta) %>% 
  arrange(scoala_de_provenienta,-n) %>% 
  slice(1) %>% 
  filter(!is.na(market)) %>% 
  rename(market.ms2=market)
data_regression<-base::merge(data_regression,missing_schools,by.x='scoala_de_provenienta',by.y="scoala_de_provenienta",all.x=T) %>% 
  mutate(market.ms=ifelse(is.na(market.ms),market.ms2,market.ms)) %>% select(-market.ms2)

```

```{r, opts.label='muted'}
#add deciles in market
perc.rank <- function(x) {
  y<-rank(x)/length(x)
  return(y)}
data_regression$entrance_perc_market<-NA
data_regression[!is.na(data_regression$media_la_admitere),]<-data_regression[!is.na(data_regression$media_la_admitere),] %>%
  group_by(an,market) %>%
  mutate(entrance_perc_market=perc.rank(media_la_admitere)) %>%
  ungroup
data_regression<-data_regression %>% mutate(dec_market=cut(entrance_perc_market, breaks = c(-Inf, 0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9, Inf), 
                                                   labels = c('1','2','3','4','5','6','7','8','9','10'), right = FALSE))

#number of students admitted in each market-yr
data_regression<-data_regression %>% group_by(market,an) %>% mutate(n_students_market_yr=n())



#add number of hs in each market
data_regression<-data_regression %>%  
  filter(!is.na(id_bac) & !is.na(id_adm)) %>% 
  group_by(market,an) %>% 
  mutate(n_hs_market=length(unique(school_harmonized)))

data_regression$n_hs_market_group<-data_regression$n_hs_market
data_regression$n_hs_market_group[data_regression$n_hs_market>=4 & data_regression$n_hs_market<=15]<-"4-15"
data_regression$n_hs_market_group[data_regression$n_hs_market>15]<-"16+"
data_regression$n_hs_market_group<-with(data_regression, reorder(n_hs_market_group, n_hs_market))
```

### Stats

```{r, opts.label='regular_cache'}
#How many students attend outside market
stat_market<-data_regression %>% ungroup() %>% mutate(outside=ifelse(market!=market.ms,T,F)) %>% filter(!is.na(outside)) %>% summarise(sum(outside)/n())
outside_students<-data_regression %>% filter(market!=market.ms) %>% group_by(market,market.ms,school_harmonized,scoala_de_provenienta) %>% summarise(n=n()) %>% arrange(school_harmonized,-n)

data_regression<-data_regression %>% mutate(migrant=ifelse(market!=market.ms,T,F))

sum(data_regression$migrant[!is.na(data_regression$migrant) & data_regression$school_change==F])/nrow(data_regression )

sum(data_regression$school_change[!is.na(data_regression$school_change)])/nrow(data_regression)
nrow(data_regression %>% filter(migrant==F & school_change==F))/nrow(data_regression %>% filter(school_change==F))

with(data_regression,table(migrant,n_hs_market_group))
with(data_regression,table(migrant,n_hs_town_group))
#with(data_regression,table(town_hs,n_hs_market_group))




```
### School Regressions
Run regression:

```{r, opts.label='regular'}
model_school_full<-feols(grad_perc~entrance_perc+dec_market+n_hs_market_group+dec_market*n_students_market_yr+Unemployment_hs_bac*dec_market+Wages_hs_bac*dec_market+drop_hs_hs_bac*dec_market|
        as.factor(an)+specializare_bac2+scoala_de_provenienta+as.factor(market)|
        school_mean~dec_market*n_hs_market_group,
      cluster=~judet_bac,
      data=data_regression %>% filter(school_change==F))
stage1_school_full<-summary(model_school_full,stage=1)
gc()

model_school_empty<-feols(grad_perc~entrance_perc+dec_market+n_hs_market_group|
        0|
        school_mean~dec_market*n_hs_market_group,
      cluster=~judet_bac,
      data=data_regression %>% filter(school_change==F))
stage1_school_empty<-summary(model_school_empty,stage=1)
gc()

model_school_controls<-feols(grad_perc~entrance_perc+dec_market+n_hs_market_group|
        as.factor(an)+specializare_bac2+scoala_de_provenienta+as.factor(market)|
        school_mean~dec_market*n_hs_market_group,
      cluster=~judet_bac,
      data=data_regression %>% filter(school_change==F))
stage1_school_controls<-summary(model_school_controls,stage=1)
gc()

summary(model_school_empty)
summary(model_school_controls)
summary(model_school_full)

summary(model_school_full,stage=1)


```

Show second stage:

```{r, opts.label='latex'}
print(modelsummary(list(model_school_empty,model_school_controls,model_school_full), 
                   statistic = "std.error",
                   estimate="{estimate}{stars}",
                   stars=c('^{*}'=0.1,'^{**}'=0.05,'^{***}'=0.01),
                   output="latex",
                   #gof_map=metrics,
                   gof_omit = 'AIC|R2 Pseudo|R2 Adj|R2 Within|BIC|Log.Lik.|Sigma',
                   coef_omit='none',
                   #coef_rename=variables,
                   fmt=2,
                   escape=F))
```
```{r,opts.label='invisible'}
#rm(model_school_empty,model_school_controls,model_school_full)
gc()
```

### School Regressions: No Migrants
Run regression:

```{r, opts.label='regular'}
model_school_full<-feols(grad_perc~entrance_perc+dec_market+n_hs_market_group+dec_market*n_students_market_yr+Unemployment_hs_bac*dec_market+Wages_hs_bac*dec_market+drop_hs_hs_bac*dec_market|
        as.factor(an)+specializare_bac2+scoala_de_provenienta+as.factor(market)|
        school_mean~dec_market*n_hs_market_group,
      cluster=~judet_bac,
      data=data_regression %>% filter(school_change==F & migrant==F))
stage1_school_full<-summary(model_school_full,stage=1)
gc()

model_school_empty<-feols(grad_perc~entrance_perc+dec_market+n_hs_market_group|
        0|
        school_mean~dec_market*n_hs_market_group,
      cluster=~judet_bac,
      data=data_regression %>% filter(school_change==F & migrant==F))
stage1_school_empty<-summary(model_school_empty,stage=1)
gc()

model_school_controls<-feols(grad_perc~entrance_perc+dec_market+n_hs_market_group|
        as.factor(an)+specializare_bac2+scoala_de_provenienta+as.factor(market)|
        school_mean~dec_market*n_hs_market_group,
      cluster=~judet_bac,
      data=data_regression %>% filter(school_change==F & migrant==F))
stage1_school_controls<-summary(model_school_controls,stage=1)
gc()

summary(model_school_empty)
summary(model_school_controls)
summary(model_school_full)


```

Show second stage:

```{r, opts.label='latex'}
print(modelsummary(list(model_school_empty,model_school_controls,model_school_full), 
                   statistic = "std.error",
                   estimate="{estimate}{stars}",
                   stars=c('^{*}'=0.1,'^{**}'=0.05,'^{***}'=0.01),
                   output="latex",
                   #gof_map=metrics,
                   gof_omit = 'AIC|R2 Pseudo|R2 Adj|R2 Within|BIC|Log.Lik.|Sigma',
                   coef_omit='none',
                   #coef_rename=variables,
                   fmt=2,
                   escape=F))
```
```{r,opts.label='invisible'}
#rm(model_school_empty,model_school_controls,model_school_full)
gc()
```

### Track Regressions
Run regression:

```{r, opts.label='regular'}
model_track_full<-feols(grad_perc~entrance_perc+dec_market+n_hs_market_group+dec_market*n_students_market_yr+Unemployment_hs_bac*dec_market+Wages_hs_bac*dec_market+drop_hs_hs_bac*dec_market|
        as.factor(an)+specializare_bac2+scoala_de_provenienta+as.factor(market)|
        class_mean~dec_market*n_hs_market_group,
      cluster=~judet_bac,
      data=data_regression %>% filter(school_change==F))
stage1_track_full<-summary(model_track_full,stage=1)
gc()

model_track_empty<-feols(grad_perc~entrance_perc+dec_market+n_hs_market_group|
        0|
        class_mean~dec_market*n_hs_market_group,
      cluster=~judet_bac,
      data=data_regression %>% filter(school_change==F))
stage1_track_empty<-summary(model_track_empty,stage=1)
gc()

model_track_controls<-feols(grad_perc~entrance_perc+dec_market+n_hs_market_group|
        as.factor(an)+specializare_bac2+scoala_de_provenienta+as.factor(market)|
        class_mean~dec_market*n_hs_market_group,
      cluster=~judet_bac,
      data=data_regression %>% filter(school_change==F))
stage1_track_controls<-summary(model_track_controls,stage=1)
gc()

summary(model_track_empty)
summary(model_track_controls)
summary(model_track_full)


```

Show second stage:

```{r, opts.label='latex'}
print(modelsummary(list(model_track_empty,model_track_controls,model_track_full), 
                   statistic = "std.error",
                   estimate="{estimate}{stars}",
                   stars=c('^{*}'=0.1,'^{**}'=0.05,'^{***}'=0.01),
                   output="latex",
                   #gof_map=metrics,
                   gof_omit = 'AIC|R2 Pseudo|R2 Adj|R2 Within|BIC|Log.Lik.|Sigma',
                   coef_omit='none',
                   #coef_rename=variables,
                   fmt=2,
                   escape=F))
```

### Track Regressions: No Migrants
Run regression:

```{r, opts.label='regular'}
model_track_full<-feols(grad_perc~entrance_perc+dec_market+n_hs_market_group+dec_market*n_students_market_yr+Unemployment_hs_bac*dec_market+Wages_hs_bac*dec_market+drop_hs_hs_bac*dec_market|
        as.factor(an)+specializare_bac2+scoala_de_provenienta+as.factor(market)|
        class_mean~dec_market*n_hs_market_group,
      cluster=~judet_bac,
      data=data_regression %>% filter(school_change==F & migrant==F))
stage1_track_full<-summary(model_track_full,stage=1)
gc()

model_track_empty<-feols(grad_perc~entrance_perc+dec_market+n_hs_market_group|
        0|
        class_mean~dec_market*n_hs_market_group,
      cluster=~judet_bac,
      data=data_regression %>% filter(school_change==F & migrant==F))
stage1_track_empty<-summary(model_track_empty,stage=1)
gc()

model_track_controls<-feols(grad_perc~entrance_perc+dec_market+n_hs_market_group|
        as.factor(an)+specializare_bac2+scoala_de_provenienta+as.factor(market)|
        class_mean~dec_market*n_hs_market_group,
      cluster=~judet_bac,
      data=data_regression %>% filter(school_change==F & migrant==F))
stage1_track_controls<-summary(model_track_controls,stage=1)
gc()

summary(model_track_empty)
summary(model_track_controls)
summary(model_track_full)


```

Show second stage:

```{r, opts.label='latex'}
print(modelsummary(list(model_track_empty,model_track_controls,model_track_full),          
                   statistic = "std.error",
                   estimate="{estimate}{stars}",
                   stars=c('^{*}'=0.1,'^{**}'=0.05,'^{***}'=0.01),
                   output="latex",
                   #gof_map=metrics,
                   gof_omit = 'AIC|R2 Pseudo|R2 Adj|R2 Within|BIC|Log.Lik.|Sigma',
                   coef_omit='none',
                   #coef_rename=variables,
                   fmt=2,
                   escape=F))
```